import re
import time

from common.utils import gen_keyword
from database.db import save_movie_list, get_categories
from detail import md5
from downloader import download


def format_url(base_url, movie_url):
    if base_url[-1] == '/':
        base_url = base_url[:-1]
    return f'{base_url}{movie_url}'


def format_base_url(base_url):
    if base_url[-1] == '/':
        base_url = base_url[:-1]
    return base_url


def parse_movie_list(category, isFirstRun=False):
    url = category['url']
    base_url = category['base_url']
    page = 0
    while url:
        if 'http' not in url:
            url = f'{format_base_url(base_url)}{url}'
        resp = download(url)
        try:
            lis = resp.xpath('//ul[@class="list"]/li')
        except:
            raise ValueError(url)
        movies = []
        for li in lis:
            movie_url = li.xpath('a/@href').extract()[0]
            movie_md5 = md5(movie_url)
            movie_url = format_url(base_url, movie_url)
            date = li.xpath('span/text()').extract()[0]
            try:
                title = li.xpath('a/font/text()').extract()[0]
                isRecommend = 1
            except:
                try:
                    title = li.xpath('a/text()').extract()[0]
                    isRecommend = 0
                except:
                    continue
            main_category, category_list, type = gen_keyword(title, movie_url)
            try:
                name = re.findall(r'《(.*)》', title)[0]
            except:
                name = title if isinstance(title, str) else title[0]
            print(name, movie_url, isRecommend)
            try:
                order = int(re.findall(r'(\d{2,8})\.html', movie_url)[0])
            except:
                order = 0

            movies.append({
                'order': order,
                'category': '/'.join(category_list),
                'date': date,
                'title': title,
                'name': name,
                'url': movie_url,
                'isRecommend': isRecommend,
                'updated': 1,
                'md5': movie_md5,
                'category_list': category_list,
                'main_category': main_category,
                'type': type

            })

        if not save_movie_list(movies):
            if not isFirstRun:
                break
        page += 1
        if page == 20:
            break
        try:
            url = resp.xpath('//a[contains(text(),"下一页")]/@href').extract()[0]
        except:
            break


def run_list_spider(isFirstRun=True):
    categories = get_categories()
    for cate in categories:
        print(cate['category'])
        parse_movie_list(cate, isFirstRun)
        time.sleep(10)


if __name__ == '__main__':
    run_list_spider()
